{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kashgari Classification Benchmarks\n",
    "\n",
    "- Kashgari: 2.0.0\n",
    "- TensorFLow: 2.0.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data and Language Models\n",
    "\n",
    "### Corpus\n",
    "\n",
    "We are using the in the [TNEWS'数据集下载](https://storage.googleapis.com/cluebenchmark/tasks/tnews_public.zip)\n",
    "in [中文任务基准测评(CLUE benchmark)](https://github.com/CLUEbenchmark/CLUE).\n",
    "\n",
    "### Language models\n",
    "\n",
    "Download Embeddings to Embddings Folder and unzip.\n",
    "- [BERT-Base, Chinese](https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip)\n",
    "\n",
    "Final folder struct is\n",
    "\n",
    "```\n",
    ".\n",
    "└── embeddings\n",
    "    └── chinese_L-12_H-768_A-12\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup macros\n",
    "EMBEDDING_FOLDER = '/Users/brikerman/Desktop/kashgari-demo/embeddings'\n",
    "EARL_STOPPING_PATIENCE = 5\n",
    "REDUCE_RL_PATIENCE = 5\n",
    "\n",
    "EPOCHS = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/cluebenchmark/tasks/tnews_public.zip\n",
      "4694016/4689325 [==============================] - 1s 0us/step\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from tensorflow.keras.utils import get_file\n",
    "from kashgari.macros import DATA_PATH\n",
    "\n",
    "# Download data to `~/.kashgari/tnews_public`\n",
    "get_file('tnews_public.zip',\n",
    "         'https://storage.googleapis.com/cluebenchmark/tasks/tnews_public.zip',\n",
    "         cache_subdir='tnews_public',\n",
    "         cache_dir=DATA_PATH,\n",
    "         extract=True)\n",
    "\n",
    "corpus_path = os.path.join(DATA_PATH, 'tnews_public')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Preprocess Dataset\n",
    "\n",
    "We will split `train.json` dataset to train and valid dataset by 8:2 rate, and use the `dev.json` as testset.\n",
    "This is because the `test.json` is unlabeled data, and we can't use it as testset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train samples : 42688\n",
      "Valid samples : 10672\n",
      "Test  samples : 10000\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from typing import List, Tuple, Dict\n",
    "from kashgari.tokenizers import BertTokenizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "tokenizer = BertTokenizer()\n",
    "\n",
    "def parse_data_file(file_path: str) -> Tuple[List[List[str]], List[str]]:\n",
    "    x_set: List[List[str]] = []\n",
    "    y_set: List[str] = []\n",
    "    with open(file_path, 'r') as f:\n",
    "        for line in f.readlines():\n",
    "            sample = json.loads(line)\n",
    "            x = tokenizer.tokenize(sample['sentence'])\n",
    "            y = sample['label_desc'].replace('news_', '')\n",
    "            x_set.append(x)\n",
    "            y_set.append(y)\n",
    "    return x_set, y_set\n",
    "\n",
    "\n",
    "train_json_x, train_json_y = parse_data_file(os.path.join(corpus_path, 'train.json'))\n",
    "test_x, test_y = parse_data_file(os.path.join(corpus_path, 'dev.json'))\n",
    "\n",
    "train_x, valid_x, train_y, valid_y = train_test_split(train_json_x, train_json_y,\n",
    "                                                      test_size=0.2,\n",
    "                                                      random_state=42)\n",
    "\n",
    "print(f'Train samples : {len(train_x)}')\n",
    "print(f'Valid samples : {len(valid_x)}')\n",
    "print(f'Test  samples : {len(test_x)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from kashgari.tasks.classification import BiGRU_Model, BiLSTM_Model\n",
    "from kashgari.tasks.classification import CNN_Model, CNN_Attention_Model\n",
    "from kashgari.tasks.classification import CNN_GRU_Model, CNN_LSTM_Model\n",
    "from kashgari.embeddings import BertEmbedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Google Bert\n",
    "bert_chinese = BertEmbedding(os.path.join(EMBEDDING_FOLDER, 'chinese_L-12_H-768_A-12'))\n",
    "\n",
    "embeddings = [\n",
    "    None,\n",
    "    bert_chinese,\n",
    "]\n",
    "\n",
    "model_classes_list = [\n",
    "    BiGRU_Model,\n",
    "    BiLSTM_Model,\n",
    "    CNN_Model,\n",
    "    CNN_Attention_Model,\n",
    "    CNN_GRU_Model,\n",
    "    CNN_LSTM_Model\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Preparing text vocab dict: 100%|██████████| 42688/42688 [00:00<00:00, 202363.82it/s]\n",
      "Preparing classification label vocab dict: 100%|██████████| 42688/42688 [00:00<00:00, 1088699.61it/s]\n",
      "Calculating sequence length: 100%|██████████| 42688/42688 [00:00<00:00, 1152884.68it/s]\n",
      "WARNING:root:Calculated sequence length = 30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input (InputLayer)           [(None, None)]            0         \n",
      "_________________________________________________________________\n",
      "layer_embedding (Embedding)  (None, None, 100)         436200    \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, 256)               176640    \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 15)                3855      \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 15)                0         \n",
      "=================================================================\n",
      "Total params: 616,695\n",
      "Trainable params: 616,695\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train for 667 steps\n",
      "667/667 [==============================] - 24s 36ms/step - loss: 1.9156 - accuracy: 0.3616\n",
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input (InputLayer)           [(None, None)]            0         \n",
      "_________________________________________________________________\n",
      "layer_embedding (Embedding)  (None, None, 100)         436200    \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, 256)               176640    \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 15)                3855      \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 15)                0         \n",
      "=================================================================\n",
      "Total params: 616,695\n",
      "Trainable params: 616,695\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train for 667 steps, validate for 166 steps\n",
      "Epoch 1/30\n",
      "  1/667 [..............................] - ETA: 26s - loss: 1.4133 - accuracy: 0.5000WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (0.176361). Check your callbacks.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (0.176361). Check your callbacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "666/667 [============================>.] - ETA: 0s - loss: 1.4603 - accuracy: 0.5213"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Sequence length is None, will use the max length of the samples, which is 127\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "  agriculture     0.4462    0.4696    0.4576       494\n",
      "          car     0.6245    0.5992    0.6116       791\n",
      "      culture     0.3319    0.6101    0.4299       736\n",
      "          edu     0.5377    0.5186    0.5280       646\n",
      "entertainment     0.5155    0.4758    0.4949       910\n",
      "      finance     0.4451    0.5042    0.4728       956\n",
      "         game     0.6018    0.4977    0.5449       659\n",
      "        house     0.6124    0.4974    0.5489       378\n",
      "     military     0.4932    0.4525    0.4720       716\n",
      "       sports     0.6491    0.6415    0.6452       767\n",
      "        stock     0.0000    0.0000    0.0000        45\n",
      "        story     0.6667    0.0093    0.0183       215\n",
      "         tech     0.5118    0.3976    0.4475      1089\n",
      "       travel     0.3529    0.4343    0.3894       693\n",
      "        world     0.4549    0.4287    0.4414       905\n",
      "\n",
      "     accuracy                         0.4861     10000\n",
      "    macro avg     0.4829    0.4358    0.4335     10000\n",
      " weighted avg     0.5036    0.4861    0.4830     10000\n",
      "\n",
      "\n",
      "epoch: 0 precision: 0.503648, recall: 0.486100, f1-score: 0.482961\n",
      "667/667 [==============================] - 29s 43ms/step - loss: 1.4604 - accuracy: 0.5212 - val_loss: 1.5404 - val_accuracy: 0.5043\n",
      "Epoch 2/30\n",
      "  1/667 [..............................] - ETA: 24s - loss: 1.2665 - accuracy: 0.6562"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "665/667 [============================>.] - ETA: 0s - loss: 1.3237 - accuracy: 0.5612               precision    recall  f1-score   support\n",
      "\n",
      "  agriculture     0.4844    0.4393    0.4607       494\n",
      "          car     0.5547    0.6410    0.5947       791\n",
      "      culture     0.4074    0.4810    0.4411       736\n",
      "          edu     0.4507    0.6084    0.5178       646\n",
      "entertainment     0.5981    0.4187    0.4926       910\n",
      "      finance     0.4142    0.5628    0.4772       956\n",
      "         game     0.6409    0.4901    0.5555       659\n",
      "        house     0.6082    0.5132    0.5567       378\n",
      "     military     0.4457    0.6131    0.5162       716\n",
      "       sports     0.6421    0.6362    0.6392       767\n",
      "        stock     0.0000    0.0000    0.0000        45\n",
      "        story     0.7143    0.0930    0.1646       215\n",
      "         tech     0.4916    0.4298    0.4586      1089\n",
      "       travel     0.3416    0.5195    0.4121       693\n",
      "        world     0.5531    0.2188    0.3135       905\n",
      "\n",
      "     accuracy                         0.4880     10000\n",
      "    macro avg     0.4898    0.4443    0.4400     10000\n",
      " weighted avg     0.5099    0.4880    0.4802     10000\n",
      "\n",
      "\n",
      "epoch: 1 precision: 0.509922, recall: 0.488000, f1-score: 0.480203\n",
      "667/667 [==============================] - 29s 43ms/step - loss: 1.3241 - accuracy: 0.5611 - val_loss: 1.5339 - val_accuracy: 0.5057\n",
      "Epoch 3/30\n",
      "665/667 [============================>.] - ETA: 0s - loss: 1.2252 - accuracy: 0.5897 ETA: 0s - loss: 1.2               precision    recall  f1-score   support\n",
      "\n",
      "  agriculture     0.4478    0.4777    0.4623       494\n",
      "          car     0.6613    0.5702    0.6124       791\n",
      "      culture     0.4101    0.4742    0.4398       736\n",
      "          edu     0.4439    0.5882    0.5060       646\n",
      "entertainment     0.6024    0.3813    0.4670       910\n",
      "      finance     0.3989    0.5178    0.4506       956\n",
      "         game     0.5915    0.5099    0.5477       659\n",
      "        house     0.4820    0.6032    0.5358       378\n",
      "     military     0.4907    0.5182    0.5041       716\n",
      "       sports     0.5713    0.6741    0.6184       767\n",
      "        stock     0.0000    0.0000    0.0000        45\n",
      "        story     0.7143    0.1163    0.2000       215\n",
      "         tech     0.4507    0.4784    0.4641      1089\n",
      "       travel     0.4102    0.4315    0.4205       693\n",
      "        world     0.5031    0.3580    0.4183       905\n",
      "\n",
      "     accuracy                         0.4879     10000\n",
      "    macro avg     0.4786    0.4466    0.4431     10000\n",
      " weighted avg     0.5008    0.4879    0.4836     10000\n",
      "\n",
      "\n",
      "epoch: 2 precision: 0.500794, recall: 0.487900, f1-score: 0.483630\n",
      "667/667 [==============================] - 28s 42ms/step - loss: 1.2257 - accuracy: 0.5895 - val_loss: 1.5583 - val_accuracy: 0.5010\n",
      "Epoch 4/30\n",
      "665/667 [============================>.] - ETA: 0s - loss: 1.1401 - accuracy: 0.6165               precision    recall  f1-score   support\n",
      "\n",
      "  agriculture     0.4680    0.4737    0.4708       494\n",
      "          car     0.6567    0.5803    0.6161       791\n",
      "      culture     0.4179    0.4878    0.4502       736\n",
      "          edu     0.5080    0.5418    0.5243       646\n",
      "entertainment     0.4951    0.4978    0.4964       910\n",
      "      finance     0.3780    0.5220    0.4385       956\n",
      "         game     0.6600    0.4476    0.5335       659\n",
      "        house     0.5946    0.5238    0.5570       378\n",
      "     military     0.5092    0.5000    0.5046       716\n",
      "       sports     0.6236    0.6415    0.6324       767\n",
      "        stock     0.0000    0.0000    0.0000        45\n",
      "        story     0.6522    0.1395    0.2299       215\n",
      "         tech     0.4333    0.5161    0.4711      1089\n",
      "       travel     0.4230    0.4242    0.4236       693\n",
      "        world     0.5099    0.3978    0.4469       905\n",
      "\n",
      "     accuracy                         0.4943     10000\n",
      "    macro avg     0.4886    0.4463    0.4530     10000\n",
      " weighted avg     0.5068    0.4943    0.4930     10000\n",
      "\n",
      "\n",
      "epoch: 3 precision: 0.506756, recall: 0.494300, f1-score: 0.492980\n",
      "667/667 [==============================] - 28s 42ms/step - loss: 1.1404 - accuracy: 0.6164 - val_loss: 1.6043 - val_accuracy: 0.5000\n",
      "Epoch 5/30\n",
      "665/667 [============================>.] - ETA: 0s - loss: 1.0587 - accuracy: 0.6434               precision    recall  f1-score   support\n",
      "\n",
      "  agriculture     0.4086    0.5202    0.4577       494\n",
      "          car     0.6089    0.6043    0.6066       791\n",
      "      culture     0.4050    0.4837    0.4409       736\n",
      "          edu     0.5057    0.5449    0.5246       646\n",
      "entertainment     0.5276    0.4099    0.4613       910\n",
      "      finance     0.4053    0.5397    0.4630       956\n",
      "         game     0.6189    0.4780    0.5394       659\n",
      "        house     0.6718    0.4603    0.5463       378\n",
      "     military     0.5040    0.4372    0.4682       716\n",
      "       sports     0.5655    0.6754    0.6156       767\n",
      "        stock     0.0000    0.0000    0.0000        45\n",
      "        story     0.6481    0.1628    0.2602       215\n",
      "         tech     0.4912    0.4380    0.4631      1089\n",
      "       travel     0.3622    0.4892    0.4162       693\n",
      "        world     0.4901    0.4122    0.4478       905\n",
      "\n",
      "     accuracy                         0.4876     10000\n",
      "    macro avg     0.4809    0.4437    0.4474     10000\n",
      " weighted avg     0.5001    0.4876    0.4855     10000\n",
      "\n",
      "\n",
      "epoch: 4 precision: 0.500118, recall: 0.487600, f1-score: 0.485501\n",
      "667/667 [==============================] - 29s 43ms/step - loss: 1.0585 - accuracy: 0.6433 - val_loss: 1.6617 - val_accuracy: 0.4955\n",
      "Epoch 6/30\n",
      "666/667 [============================>.] - ETA: 0s - loss: 0.9768 - accuracy: 0.6717               precision    recall  f1-score   support\n",
      "\n",
      "  agriculture     0.4704    0.4028    0.4340       494\n",
      "          car     0.5702    0.6005    0.5850       791\n",
      "      culture     0.4145    0.4579    0.4351       736\n",
      "          edu     0.5182    0.5294    0.5237       646\n",
      "entertainment     0.4956    0.4330    0.4622       910\n",
      "      finance     0.3637    0.5931    0.4509       956\n",
      "         game     0.5881    0.5266    0.5556       659\n",
      "        house     0.6097    0.5000    0.5494       378\n",
      "     military     0.4826    0.5042    0.4932       716\n",
      "       sports     0.5821    0.6284    0.6044       767\n",
      "        stock     0.0000    0.0000    0.0000        45\n",
      "        story     0.6667    0.1488    0.2433       215\n",
      "         tech     0.4738    0.4399    0.4562      1089\n",
      "       travel     0.3939    0.4315    0.4118       693\n",
      "        world     0.5105    0.3492    0.4147       905\n",
      "\n",
      "     accuracy                         0.4819     10000\n",
      "    macro avg     0.4760    0.4363    0.4413     10000\n",
      " weighted avg     0.4926    0.4819    0.4788     10000\n",
      "\n",
      "\n",
      "epoch: 5 precision: 0.492634, recall: 0.481900, f1-score: 0.478768\n",
      "667/667 [==============================] - 27s 41ms/step - loss: 0.9765 - accuracy: 0.6718 - val_loss: 1.7340 - val_accuracy: 0.4891\n",
      "Epoch 7/30\n",
      "665/667 [============================>.] - ETA: 0s - loss: 0.8970 - accuracy: 0.6981               precision    recall  f1-score   support\n",
      "\n",
      "  agriculture     0.4408    0.4372    0.4390       494\n",
      "          car     0.6099    0.5613    0.5846       791\n",
      "      culture     0.4301    0.4429    0.4364       736\n",
      "          edu     0.5007    0.5480    0.5233       646\n",
      "entertainment     0.4632    0.4912    0.4768       910\n",
      "      finance     0.4040    0.4885    0.4422       956\n",
      "         game     0.6521    0.4522    0.5341       659\n",
      "        house     0.5821    0.5344    0.5572       378\n",
      "     military     0.4737    0.5154    0.4936       716\n",
      "       sports     0.6609    0.5997    0.6288       767\n",
      "        stock     0.0000    0.0000    0.0000        45\n",
      "        story     0.5862    0.1581    0.2491       215\n",
      "         tech     0.4546    0.4784    0.4662      1089\n",
      "       travel     0.3389    0.5267    0.4124       693\n",
      "        world     0.5142    0.3602    0.4237       905\n",
      "\n",
      "     accuracy                         0.4829     10000\n",
      "    macro avg     0.4741    0.4396    0.4445     10000\n",
      " weighted avg     0.4965    0.4829    0.4824     10000\n",
      "\n",
      "\n",
      "epoch: 6 precision: 0.496509, recall: 0.482900, f1-score: 0.482403\n",
      "667/667 [==============================] - 27s 40ms/step - loss: 0.8967 - accuracy: 0.6982 - val_loss: 1.8611 - val_accuracy: 0.4758\n",
      "Epoch 8/30\n",
      "665/667 [============================>.] - ETA: 0s - loss: 0.7147 - accuracy: 0.7701               precision    recall  f1-score   support\n",
      "\n",
      "  agriculture     0.4219    0.4595    0.4399       494\n",
      "          car     0.5915    0.5803    0.5858       791\n",
      "      culture     0.4145    0.4511    0.4320       736\n",
      "          edu     0.5293    0.5310    0.5301       646\n",
      "entertainment     0.4923    0.4571    0.4741       910\n",
      "      finance     0.3983    0.4958    0.4418       956\n",
      "         game     0.6056    0.4962    0.5455       659\n",
      "        house     0.5981    0.5000    0.5447       378\n",
      "     military     0.4778    0.4818    0.4798       716\n",
      "       sports     0.5889    0.6519    0.6188       767\n",
      "        stock     0.2000    0.0222    0.0400        45\n",
      "        story     0.6170    0.1349    0.2214       215\n",
      "         tech     0.4491    0.4784    0.4633      1089\n",
      "       travel     0.3791    0.4026    0.3905       693\n",
      "        world     0.4631    0.4232    0.4423       905\n",
      "\n",
      "     accuracy                         0.4825     10000\n",
      "    macro avg     0.4818    0.4377    0.4433     10000\n",
      " weighted avg     0.4884    0.4825    0.4803     10000\n",
      "\n",
      "\n",
      "epoch: 7 precision: 0.488368, recall: 0.482500, f1-score: 0.480320\n",
      "667/667 [==============================] - 27s 41ms/step - loss: 0.7141 - accuracy: 0.7704 - val_loss: 1.8951 - val_accuracy: 0.4870\n",
      "Epoch 9/30\n",
      "666/667 [============================>.] - ETA: 0s - loss: 0.6795 - accuracy: 0.7827               precision    recall  f1-score   support\n",
      "\n",
      "  agriculture     0.4100    0.4656    0.4360       494\n",
      "          car     0.5884    0.5765    0.5824       791\n",
      "      culture     0.4113    0.4443    0.4272       736\n",
      "          edu     0.5227    0.5356    0.5291       646\n",
      "entertainment     0.4934    0.4538    0.4728       910\n",
      "      finance     0.3924    0.4958    0.4381       956\n",
      "         game     0.6045    0.4917    0.5423       659\n",
      "        house     0.5888    0.5000    0.5408       378\n",
      "     military     0.4809    0.4735    0.4771       716\n",
      "       sports     0.5830    0.6545    0.6167       767\n",
      "        stock     0.1429    0.0222    0.0385        45\n",
      "        story     0.6471    0.1535    0.2481       215\n",
      "         tech     0.4517    0.4812    0.4660      1089\n",
      "       travel     0.3954    0.4012    0.3983       693\n",
      "        world     0.4658    0.4210    0.4423       905\n",
      "\n",
      "     accuracy                         0.4817     10000\n",
      "    macro avg     0.4785    0.4380    0.4437     10000\n",
      " weighted avg     0.4878    0.4817    0.4796     10000\n",
      "\n",
      "\n",
      "epoch: 8 precision: 0.487789, recall: 0.481700, f1-score: 0.479646\n",
      "667/667 [==============================] - 30s 45ms/step - loss: 0.6793 - accuracy: 0.7827 - val_loss: 1.9317 - val_accuracy: 0.4827\n",
      "Epoch 10/30\n",
      "306/667 [============>.................] - ETA: 14s - loss: 0.7286 - accuracy: 0.7633WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss,accuracy\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss,accuracy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Reduce LR on plateau conditioned on metric `val_loss` which is not available. Available metrics are: loss,accuracy,lr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Reduce LR on plateau conditioned on metric `val_loss` which is not available. Available metrics are: loss,accuracy,lr\n",
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n",
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 681, in on_epoch\n",
      "    yield epoch_logs\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 324, in fit\n",
      "    total_epochs=epochs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 123, in run_one_epoch\n",
      "    batch_outs = execution_function(iterator)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\", line 86, in execution_function\n",
      "    distributed_function(input_fn))\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\", line 457, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\", line 487, in _call\n",
      "    return self._stateless_fn(*args, **kwds)  # pylint: disable=not-callable\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1823, in __call__\n",
      "    return graph_function._filtered_call(args, kwargs)  # pylint: disable=protected-access\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1141, in _filtered_call\n",
      "    self.captured_inputs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1224, in _call_flat\n",
      "    ctx, args, cancellation_manager=cancellation_manager)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 511, in call\n",
      "    ctx=ctx)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py\", line 61, in quick_execute\n",
      "    num_outputs)\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3331, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-6-aa5ce7af04b5>\", line 50, in <module>\n",
      "    epochs=EPOCHS)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/kashgari/tasks/classification/abc_model.py\", line 124, in fit\n",
      "    fit_kwargs=fit_kwargs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/kashgari/tasks/classification/abc_model.py\", line 187, in fit_generator\n",
      "    **fit_kwargs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py\", line 728, in fit\n",
      "    use_multiprocessing=use_multiprocessing)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 372, in fit\n",
      "    prefix='val_')\n",
      "  File \"/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/contextlib.py\", line 130, in __exit__\n",
      "    self.gen.throw(type, value, traceback)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 685, in on_epoch\n",
      "    self.callbacks.on_epoch_end(epoch, epoch_logs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/callbacks.py\", line 298, in on_epoch_end\n",
      "    callback.on_epoch_end(epoch, logs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/kashgari/callbacks/eval_callBack.py\", line 51, in on_epoch_end\n",
      "    batch_size=self.batch_size)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/kashgari/tasks/classification/abc_model.py\", line 259, in evaluate\n",
      "    debug_info=debug_info)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/kashgari/tasks/classification/abc_model.py\", line 225, in predict\n",
      "    pred = self.tf_model.predict(tensor, batch_size=batch_size, **predict_kwargs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py\", line 909, in predict\n",
      "    use_multiprocessing=use_multiprocessing)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 462, in predict\n",
      "    steps=steps, callbacks=callbacks, **kwargs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 444, in _model_iteration\n",
      "    total_epochs=1)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 123, in run_one_epoch\n",
      "    batch_outs = execution_function(iterator)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\", line 86, in execution_function\n",
      "    distributed_function(input_fn))\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\", line 457, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\", line 494, in _call\n",
      "    results = self._stateful_fn(*args, **kwds)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1823, in __call__\n",
      "    return graph_function._filtered_call(args, kwargs)  # pylint: disable=protected-access\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1141, in _filtered_call\n",
      "    self.captured_inputs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1224, in _call_flat\n",
      "    ctx, args, cancellation_manager=cancellation_manager)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 511, in call\n",
      "    ctx=ctx)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py\", line 61, in quick_execute\n",
      "    num_outputs)\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1148, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 316, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 350, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 1502, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 1460, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 696, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 742, in getmodule\n",
      "    os.path.realpath(f)] = module.__name__\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/bin/../lib/python3.7/posixpath.py\", line 395, in realpath\n",
      "    path, ok = _joinrealpath(filename[:0], filename, {})\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/bin/../lib/python3.7/posixpath.py\", line 429, in _joinrealpath\n",
      "    if not islink(newpath):\n",
      "KeyboardInterrupt\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 681, in on_epoch\n",
      "    yield epoch_logs\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 324, in fit\n",
      "    total_epochs=epochs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 123, in run_one_epoch\n",
      "    batch_outs = execution_function(iterator)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\", line 86, in execution_function\n",
      "    distributed_function(input_fn))\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\", line 457, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\", line 487, in _call\n",
      "    return self._stateless_fn(*args, **kwds)  # pylint: disable=not-callable\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1823, in __call__\n",
      "    return graph_function._filtered_call(args, kwargs)  # pylint: disable=protected-access\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1141, in _filtered_call\n",
      "    self.captured_inputs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1224, in _call_flat\n",
      "    ctx, args, cancellation_manager=cancellation_manager)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 511, in call\n",
      "    ctx=ctx)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py\", line 61, in quick_execute\n",
      "    num_outputs)\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3331, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-6-aa5ce7af04b5>\", line 50, in <module>\n",
      "    epochs=EPOCHS)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/kashgari/tasks/classification/abc_model.py\", line 124, in fit\n",
      "    fit_kwargs=fit_kwargs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/kashgari/tasks/classification/abc_model.py\", line 187, in fit_generator\n",
      "    **fit_kwargs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py\", line 728, in fit\n",
      "    use_multiprocessing=use_multiprocessing)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 372, in fit\n",
      "    prefix='val_')\n",
      "  File \"/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/contextlib.py\", line 130, in __exit__\n",
      "    self.gen.throw(type, value, traceback)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 685, in on_epoch\n",
      "    self.callbacks.on_epoch_end(epoch, epoch_logs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/callbacks.py\", line 298, in on_epoch_end\n",
      "    callback.on_epoch_end(epoch, logs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/kashgari/callbacks/eval_callBack.py\", line 51, in on_epoch_end\n",
      "    batch_size=self.batch_size)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/kashgari/tasks/classification/abc_model.py\", line 259, in evaluate\n",
      "    debug_info=debug_info)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/kashgari/tasks/classification/abc_model.py\", line 225, in predict\n",
      "    pred = self.tf_model.predict(tensor, batch_size=batch_size, **predict_kwargs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py\", line 909, in predict\n",
      "    use_multiprocessing=use_multiprocessing)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 462, in predict\n",
      "    steps=steps, callbacks=callbacks, **kwargs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 444, in _model_iteration\n",
      "    total_epochs=1)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\", line 123, in run_one_epoch\n",
      "    batch_outs = execution_function(iterator)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\", line 86, in execution_function\n",
      "    distributed_function(input_fn))\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\", line 457, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\", line 494, in _call\n",
      "    results = self._stateful_fn(*args, **kwds)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1823, in __call__\n",
      "    return graph_function._filtered_call(args, kwargs)  # pylint: disable=protected-access\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1141, in _filtered_call\n",
      "    self.captured_inputs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 1224, in _call_flat\n",
      "    ctx, args, cancellation_manager=cancellation_manager)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\", line 511, in call\n",
      "    ctx=ctx)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py\", line 61, in quick_execute\n",
      "    num_outputs)\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3254, in run_ast_nodes\n",
      "    if (await self.run_code(code, result,  async_=asy)):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3348, in run_code\n",
      "    self.showtraceback(running_compiled_code=True)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2047, in showtraceback\n",
      "    value, tb, tb_offset=tb_offset)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1415, in structured_traceback\n",
      "    self, etype, value, tb, tb_offset, number_of_lines_of_context)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1315, in structured_traceback\n",
      "    self, etype, value, tb, tb_offset, number_of_lines_of_context\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1183, in structured_traceback\n",
      "    formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__)\n",
      "TypeError: can only concatenate str (not \"list\") to str\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'TypeError' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1148, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 316, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 350, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 1502, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 1460, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 696, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 742, in getmodule\n",
      "    os.path.realpath(f)] = module.__name__\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/bin/../lib/python3.7/posixpath.py\", line 395, in realpath\n",
      "    path, ok = _joinrealpath(filename[:0], filename, {})\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/bin/../lib/python3.7/posixpath.py\", line 429, in _joinrealpath\n",
      "    if not islink(newpath):\n",
      "  File \"/Users/brikerman/Desktop/python/Kashgari2/venv/bin/../lib/python3.7/posixpath.py\", line 171, in islink\n",
      "    st = os.lstat(path)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "can only concatenate str (not \"list\") to str",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mon_epoch\u001b[0;34m(self, epoch, mode)\u001b[0m\n\u001b[1;32m    680\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 681\u001b[0;31m       \u001b[0;32myield\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    682\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, **kwargs)\u001b[0m\n\u001b[1;32m    323\u001b[0m                 \u001b[0mtraining_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m                 total_epochs=epochs)\n\u001b[0m\u001b[1;32m    325\u001b[0m             \u001b[0mcbks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_logs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mrun_one_epoch\u001b[0;34m(model, iterator, execution_function, dataset_size, batch_size, strategy, steps_per_epoch, num_samples, mode, training_context, total_epochs)\u001b[0m\n\u001b[1;32m    122\u001b[0m       \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 123\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    124\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mStopIteration\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOutOfRangeError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\u001b[0m in \u001b[0;36mexecution_function\u001b[0;34m(input_fn)\u001b[0m\n\u001b[1;32m     85\u001b[0m     return nest.map_structure(_non_none_constant_value,\n\u001b[0;32m---> 86\u001b[0;31m                               distributed_function(input_fn))\n\u001b[0m\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    456\u001b[0m     \u001b[0mtracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 457\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    458\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    486\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 487\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    488\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1822\u001b[0m     \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1823\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1824\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   1140\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[0;32m-> 1141\u001b[0;31m         self.captured_inputs)\n\u001b[0m\u001b[1;32m   1142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1223\u001b[0m       flat_outputs = forward_function.call(\n\u001b[0;32m-> 1224\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[1;32m   1225\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    510\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"executor_type\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"config_proto\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 511\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    512\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m                                                num_outputs)\n\u001b[0m\u001b[1;32m     62\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mshowtraceback\u001b[0;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[1;32m   2043\u001b[0m                         \u001b[0;31m# in the engines. This should return a list of strings.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2044\u001b[0;31m                         \u001b[0mstb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_render_traceback_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2045\u001b[0m                     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'KeyboardInterrupt' object has no attribute '_render_traceback_'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_code\u001b[0;34m(self, code_obj, result, async_)\u001b[0m\n\u001b[1;32m   3346\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3347\u001b[0m                 \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror_in_exec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3348\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshowtraceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrunning_compiled_code\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3349\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3350\u001b[0m             \u001b[0moutflag\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mshowtraceback\u001b[0;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[1;32m   2045\u001b[0m                     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2046\u001b[0m                         stb = self.InteractiveTB.structured_traceback(etype,\n\u001b[0;32m-> 2047\u001b[0;31m                                             value, tb, tb_offset=tb_offset)\n\u001b[0m\u001b[1;32m   2048\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2049\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_showtraceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1413\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1414\u001b[0m         return FormattedTB.structured_traceback(\n\u001b[0;32m-> 1415\u001b[0;31m             self, etype, value, tb, tb_offset, number_of_lines_of_context)\n\u001b[0m\u001b[1;32m   1416\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1417\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1313\u001b[0m             \u001b[0;31m# Verbose modes need a full traceback\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1314\u001b[0m             return VerboseTB.structured_traceback(\n\u001b[0;32m-> 1315\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_offset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnumber_of_lines_of_context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1316\u001b[0m             )\n\u001b[1;32m   1317\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'Minimal'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/python/Kashgari2/venv/lib/python3.7/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, evalue, etb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1181\u001b[0m         \u001b[0mexception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_parts_of_chained_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1183\u001b[0;31m             \u001b[0mformatted_exceptions\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare_chained_exception_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__cause__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1184\u001b[0m             \u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0metb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1185\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: can only concatenate str (not \"list\") to str"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from kashgari.callbacks import EvalCallBack\n",
    "from benchmark_utils import  BenchMarkHelper\n",
    "\n",
    "run_count = glob.glob('./tf_dir/classification/run_*')\n",
    "TF_LOG_FOLDER = f'./tf_dir/classification/run_{len(run_count)}'\n",
    "TRAINING_LOG = f'./training_logs_{len(run_count)}.json'\n",
    "\n",
    "\n",
    "for embed in embeddings:\n",
    "    for MODEL_CLASS in model_classes_list:\n",
    "        model_name = MODEL_CLASS.__name__\n",
    "        if embed:\n",
    "            embed_name = embed.__class__.__name__\n",
    "        else:\n",
    "            embed_name = 'Bare'\n",
    "        run_name = f\"{embed_name}-{model_name}\"\n",
    "        \n",
    "        start_at = time.time()\n",
    "        \n",
    "        model = MODEL_CLASS(embed)\n",
    "        model.fit(train_x, train_y, epochs=1)\n",
    "\n",
    "        early_stop = keras.callbacks.EarlyStopping(patience=EARL_STOPPING_PATIENCE)\n",
    "        reduse_lr_callback = keras.callbacks.ReduceLROnPlateau(factor=0.1,\n",
    "                                                               patience=REDUCE_RL_PATIENCE)\n",
    "\n",
    "        eval_callback = EvalCallBack(model,\n",
    "                                     test_x,\n",
    "                                     test_y,\n",
    "                                     step=1)\n",
    "\n",
    "        tf_board = keras.callbacks.TensorBoard(\n",
    "            log_dir=os.path.join(TF_LOG_FOLDER, run_name),\n",
    "        )\n",
    "        \n",
    "        file_writer = tf.summary.create_file_writer(os.path.join(TF_LOG_FOLDER, run_name))\n",
    "        file_writer.set_as_default()\n",
    "\n",
    "        callbacks = [early_stop, reduse_lr_callback, eval_callback, tf_board]\n",
    "\n",
    "        model.fit(train_x,\n",
    "                  train_y,\n",
    "                  valid_x,\n",
    "                  valid_y,\n",
    "                  callbacks=callbacks,\n",
    "                  epochs=EPOCHS)\n",
    "        \n",
    "        BenchMarkHelper.save_training_logs(TRAINING_LOG,\n",
    "                                           embedding_name=embed_name,\n",
    "                                           model_name=model_name,\n",
    "                                           logs=eval_callback.logs,\n",
    "                                           training_duration=time.time()-start_at)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
